package rmmk.algorithms.preprocessing.preprocessors;

import java.util.Iterator;
import java.util.List;

import rmmk.algorithms.features.global.GlobalWordCounts;
import rmmk.framework.datasources.Document;

public class RemoveMostCommonWordsFromText {

	public void process(List<Document> documents) {
		for (Document d : documents) {
			d.setTEXT(d.getTEXT().toLowerCase());
			for (String unwanted : common()) {
				d.setTEXT(d.getTEXT().replaceAll("\\s+" + unwanted + "\\s+", " "));
			}

			Iterator<String> iter = d.getExtractedWords().iterator();
			while (iter.hasNext()) {
				if (equalsAny(iter.next()))
					iter.remove();
			}
		}

	}

	private boolean equalsAny(String word) {
		for (int i = 0; i < common().length; ++i)
			if (common()[i].equalsIgnoreCase(word))
				return true;
		return false;
	}

	private String[] common() {
		return GlobalWordCounts.getGlobalWords();
	}
}
